

In [2]:

    
%matplotlib inline
import matplotlib.pyplot as pl
import numpy as np

# Some nice default configuration for plots
pl.rcParams['figure.figsize'] = 10, 7.5
pl.rcParams['axes.grid'] = True
pl.gray()









    





<matplotlib.figure.Figure at 0x10b8df390>

IPython.parallel



In [7]:

    
from IPython.parallel import Client
client = Client()



In [8]:

    
len(client)









    Out[8]:





4



In [9]:

    
%px print("Hello from the cluster engines!")









    



[stdout:0] Hello from the cluster engines!
[stdout:1] Hello from the cluster engines!
[stdout:2] Hello from the cluster engines!
[stdout:3] Hello from the cluster engines!



In [10]:

    
def where_am_i():
    import os
    import socket
    
    return "In process with pid {0} on host: '{1}'".format(
        os.getpid(), socket.gethostname())



In [11]:

    
where_am_i()









    Out[11]:





"In process with pid 78301 on host: 'host.local'"

Direct View



In [12]:

    
direct_view = client.direct_view()



In [13]:

    
where_am_i_direct_results = direct_view.apply(where_am_i)
where_am_i_direct_results









    Out[13]:





<AsyncResult: where_am_i>



In [14]:

    
where_am_i_direct_results.get()









    Out[14]:





["In process with pid 78307 on host: 'host.local'",
 "In process with pid 78308 on host: 'host.local'",
 "In process with pid 78309 on host: 'host.local'",
 "In process with pid 78310 on host: 'host.local'"]



In [15]:

    
where_am_i_direct_results.get_dict()









    Out[15]:





{0: "In process with pid 78307 on host: 'host.local'",
 1: "In process with pid 78308 on host: 'host.local'",
 2: "In process with pid 78309 on host: 'host.local'",
 3: "In process with pid 78310 on host: 'host.local'"}

Load Balanced View



In [16]:

    
lb_view = client.load_balanced_view()



In [17]:

    
where_am_i_lb_result = lb_view.apply(where_am_i)
where_am_i_lb_result









    Out[17]:





<AsyncResult: where_am_i>



In [18]:

    
where_am_i_lb_result.get()









    Out[18]:





"In process with pid 78309 on host: 'host.local'"

Distributed Grid Search for a Linear Support Vector Machine



In [19]:

    
from pyrallel import mmap_utils, model_selection
_ = reload(mmap_utils), reload(model_selection)



In [20]:

    
from sklearn.datasets import load_digits
from sklearn.preprocessing import MinMaxScaler

digits = load_digits()

X = MinMaxScaler().fit_transform(digits.data)
y = digits.target



In [21]:

    
digits_cv_split_filenames = mmap_utils.persist_cv_splits(
    X, y, name='digits_10', n_cv_iter=10)

digits_cv_split_filenames









    Out[21]:





['/Users/ogrisel/code/pyrallel/examples/digits_10_cv_000.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_001.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_002.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_003.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_004.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_005.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_006.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_007.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_008.pkl',
 '/Users/ogrisel/code/pyrallel/examples/digits_10_cv_009.pkl']



In [22]:

    
mmap_utils.warm_mmap_on_cv_splits(client, digits_cv_split_filenames)



In [23]:

    
from sklearn.svm import LinearSVC
from collections import OrderedDict
import numpy as np

linear_svc_params = OrderedDict((
    ('C', np.logspace(-2, 2, 5)),
))
linear_svc = LinearSVC()



In [24]:

    
linear_svc_search = model_selection.RandomizedGridSeach(lb_view)

linear_svc_search.launch_for_splits(
    linear_svc,
    linear_svc_params,
    digits_cv_split_filenames)









    Out[24]:





Progress: 00% (000/050)



In [25]:

    
linear_svc_search.boxplot_parameters(display_train=False)

Scaling Non-Linear SVM: Kernel Approximations

Motivation: traditional kernel SVM as in SVC has almost cubic complexity w.r.t. `n_samples`



In [26]:

    
x = np.linspace(0, int(1e3), 100)

pl.plot(x, x ** 3 / 1e9)
pl.xlabel("Number of training samples")
pl.ylabel("Estimated Convergence Time of SMO (in seconds)")









    Out[26]:





<matplotlib.text.Text at 0x10e446f50>



In [27]:

    
1e6 ** 3 / 1e9 / 60 / 60 / 24 / 365









    Out[27]:





31.709791983764582

Approximate Kernel SVM with a Explicit Non-Linear Kernel Expansion + Linear SVC



In [28]:

    
from sklearn.kernel_approximation import Nystroem
from sklearn.pipeline import Pipeline

nystroem_pipeline = Pipeline([
    ('nystroem', Nystroem()),
    ('clf', LinearSVC()),
])



In [29]:

    
nystroem_pipeline_params = OrderedDict((
    ('nystroem__n_components', [50, 100, 200]),
    ('nystroem__gamma', np.logspace(-2, 2, 5)),
    ('clf__C', np.logspace(-2, 2, 5)),
))



In [30]:

    
nystroem_search = model_selection.RandomizedGridSeach(lb_view)



In [31]:

    
nystroem_search.launch_for_splits(nystroem_pipeline, nystroem_pipeline_params, digits_cv_split_filenames)









    Out[31]:





Progress: 00% (004/750)



In [34]:

    
nystroem_search









    Out[34]:





Progress: 04% (030/750)

Rank 1: validation: 0.92578 (+/-0.00567) train: 0.93875 (+/-0.00474):
 {'nystroem__n_components': 100, 'clf__C': 10.0, 'nystroem__gamma': 1.0}
Rank 2: validation: 0.89089 (+/-0.00921) train: 0.90134 (+/-0.00597):
 {'nystroem__n_components': 100, 'clf__C': 0.10000000000000001, 'nystroem__gamma': 1.0}
Rank 3: validation: 0.08156 (+/-0.00295) train: 0.14068 (+/-0.00078):
 {'nystroem__n_components': 50, 'clf__C': 10.0, 'nystroem__gamma': 100.0}



In [35]:

    
nystroem_search.boxplot_parameters()



In [ ]:

    
nystroem_search.reset()

A Word of Caution on the Scalability of this Implementation Nystroem method

In this example we used LinearSVC that does not provide a partial_fit method hence require to put the Nystroem expansion of complet dataset in memory. Furthermore the Pipeline object does not optimize the memory usage.

To make this example really scalable we would need to:

Replace LinearSVC with an incremental linear model: Perceptron, PassiveAggressiveClassifier, SGDClassifier
Add support for memory efficient partial_fit to sklearn.pipeline.Pipeline
Change our IPython.parallel based model evaluator to use the partial_fit method with small minibatches in the inner model evaluation function.